[爬虫] python3分析《战狼2》豆瓣影评

from urllib import request
from bs4 import BeautifulSoup as bs
requrl="https://movie.douban.com/subject/26363254/comments?start=0&limit=20"
resp = request.urlopen(requrl)
html_data = resp.read().decode('utf-8')
soup = bs(html_data, 'html.parser')
comment_div_lits = soup.find_all('div', class_='comment')   #找出标签div的评论
eachCommentList = [];   #用来装载评论的空列表
for item in comment_div_lits:
        if item.find_all('p')[0].string is not None:   #找出标签p的评论
            eachCommentList.append(item.find_all('p')[0].string)
print(eachCommentList)

#爬取12万条评论
from bs4 import BeautifulSoup as bs
from selenium import webdriver
driver = webdriver.Chrome('F:\E盘文件\重要软件\chrome_drive\chromedriver_win32\chromedriver.exe')   #这里是你的浏览器驱动exe文件存储位置
#driver = webdriver.PhantomJS(executable_path="C:\\Users\\Administrator\\AppData\\Local\\Programs\\Python\\Python36\\Scripts\\phantomjs.exe")
driver.maximize_window()
driver.get("https://movie.douban.com/subject/26363254/comments?start=0&limit=20")
driver.implicitly_wait(20)
#登录
#这里需要打开浏览器界面，手动登录你的豆瓣账号,否则下文会提示你没有登录没有权限
eachCommentList = [];i=0
#控制循环条件
while i<10000:
    new_page = driver.page_source
    # 对新的页面提取该页面评论文本
    soup = bs(new_page, 'html.parser')
    comment_div_lits = soup.find_all('div', class_='comment')
    for item in comment_div_lits:
        if item.find_all('p')[0].string is not None:
            eachCommentList.append(item.find_all('p')[0].string)
    #print(eachCommentList)
    # 点击下一页
    element2 = driver.find_element_by_xpath("//a[@class='next']")
    sentinel=driver.current_url
    element2.click()
    if sentinel== driver.current_url or i>10000:
        st=input()
        if st== 'berak':
            break
    i=i+1
    print(driver.current_url)
#这里可以添加一行代码，选择保存评论文本数据

import re
import jieba  # 分词包
import pandas as pd
import numpy    #numpy计算包
def clean(result,stopwords):
    comments = ''
    for k in range(len(result)):
        comments = comments + (str(result[k])).strip()
    pattern = re.compile(r'[\u4e00-\u9fa5]+')
    filterdata = re.findall(pattern, comments)
    cleaned_comments = ''.join(filterdata)
    segment = jieba.lcut(cleaned_comments)
    words_df = pd.DataFrame({'segment': segment})
    #需要下载一个停词txt文件，过滤掉语气词等等
    stopwords = pd.read_csv(stopwords, index_col=False, quoting=3, sep="\t", names=['stopword'],
                            encoding='utf-8')  # quoting=3全不引用
    words_df = words_df[~words_df.segment.isin(stopwords.stopword)]
    return words_df
stopwords = 'E:\\blog\\assets\\stopwords-list\\stop_words_zh_UTF-82.txt'   #这里可以从其他地方导入停词的文本
result2 = clean(result=eachCommentList, stopwords=stopwords)
words_stat = result2.groupby(by=['segment'])['segment'].agg({"计数": numpy.size})
words_stat = words_stat.reset_index().sort_values(by=["计数"], ascending=False)  # 词汇统计

#使用R的词云
import rpy2.robjects as robjects
from rpy2.robjects import pandas2ri   #需要导入pandas2ri模块，并使其激活才能转换pandas数据框为R的数据框
pandas2ri.activate()
mydata=robjects.r['data.frame'](x=words_stat["segment"],y=words_stat["计数"])
# 在R进程里面执行R脚本
robjects.globalenv['mydata']=mydata
robjects.r('''
f=function(mydata) {
library(wordcloud)
n=ifelse(nrow(mydata)>200,200,nrow(mydata))
my_cloud=wordcloud(mydata[1:n,1],mydata[1:n,2], random.color = T,colors = c("purple","sienna","springgreen",
"tomato","violetred4","seagreen3"))
#my_path  <- htmltools::html_print(my_cloud)   # saves html in temp directory
#print(my_path)  # the location of the html file.
}
#path=f(mydata)
#class(mydata)
#print(mydata[1:10,1])
library(wordcloud)
f(mydata)    #在我的电脑上可以自动打开device窗口
''')